package util.localLaunchers;

import java.io.IOException;
import java.util.Enumeration;
import java.util.Hashtable;
import java.util.Iterator;
import java.util.LinkedList;

import model.Collocation;

import org.apache.pig.data.BagFactory;
import org.apache.pig.data.DataBag;
import org.apache.pig.data.DataType;
import org.apache.pig.data.Tuple;
import org.apache.pig.data.TupleFactory;
import org.apache.pig.impl.logicalLayer.schema.Schema;

import udf.string.CleanTag;
import util.io.FileInput;

public class GenerateQueryTagsLocal {

	
	BagFactory mBagFactory = BagFactory.getInstance();
	CleanTag cleaner = null;

	private boolean initialized = false;

	private Hashtable<String, Collocation> hyphenized = new Hashtable<String, Collocation>();
	//private Hashtable<String, String> simple = new Hashtable<String, String>();

	// static ArrayList<String> mTables = new ArrayList<String>();

	// static private final String domain_path =
	// "/user/duartes/helpers/domains.txt";

	/**
	 * Tuple fields
	 * 
	 * 0 set of tags of the bookmark 1 set of tags hpyhenized (dictionary)
	 * 
	 * 
	 * @param tuple
	 * @throws IOException
	 */

	/**
	 * Position 0 : a_b Position 1 : freq
	 * 
	 * 
	 */

	

	
	
	/**
	 * Improvement: ADD PMI RATIO THRESHOLD TO DECIDE WHEN TO ADD BIGRAM
	 * OR WHEN TO ADD BIGRAM AND ADD SINGLE TOKENS AS WELL TO THE HASH
	 * 
	 * 
	 * 
	 * @param query
	 * @param title
	 * @param description
	 * @param threshold
	 * @return
	 */
	
	public Hashtable<String, Integer> getTags(String query, String title,
			String description, int threshold) {
		Hashtable<String, Integer> suggestions = new Hashtable<String, Integer>();

		
		//System.out.println("Tags of  query:" +query);
		LinkedList<String> temporal = splitText(query);
		//for(int i =0; i< temporal.size(); i++){
	
		//	System.out.println(temporal.get(i));
		//}
		updateHash(suggestions, temporal, threshold);

		//System.out.println("Hash frequency:");
		//printHash(suggestions);
		
		
		temporal = splitText(title);

		//System.out.println("Tags of  title:" +title);
		
		///for(int i =0; i< temporal.size(); i++){
	
		//	System.out.println(temporal.get(i));
		//}
		//System.out.println("Hash frequency:");
		
		updateHash(suggestions, temporal, threshold);
				
		//printHash(suggestions);
		
		

		temporal = splitText(description);

		updateHash(suggestions, temporal, threshold);

		return suggestions;
	}
	
	
	public void printHash(Hashtable<String, Integer> suggestions){
		
		Enumeration<String> keys = suggestions.keys();
		
		while(keys.hasMoreElements()){
			String key = keys.nextElement();
			System.out.println(key + "\t"+ suggestions.get(key));
			
		}
	}

	/**
	 * try to match bigrams in text
	 * 
	 * if bigram is match it is added directly
	 * 
	 * otherwise each term is added independently
	 * 
	 * @param suggestions
	 * @param temporal
	 */

	private void updateHash(Hashtable<String, Integer> suggestions,
			LinkedList<String> temporal, int THRESHOLD) {
		// TODO Auto-generated method stub
		String a = null;
		String b = null;

		for (int i = 0; i < temporal.size() - 2; i++) {
			a = temporal.get(i);

			b = temporal.get(i + 1);
			int freq = 1;

			// try bigrams
			String a_b = a + "_" + b;
			a_b= a_b.toLowerCase();

			if (hyphenized.containsKey(a_b)
					&& hyphenized.get(a_b).freq_h >= THRESHOLD) {

				if (suggestions.containsKey(a_b)) {
					freq = suggestions.get(a_b) + 1;
				}

				suggestions.put(a_b, freq);

			} else {

				if (suggestions.containsKey(a)) {
					freq = suggestions.get(a) + 1;

				}
				suggestions.put(a, freq);

			}

		}
		int freq = 1;
		if (temporal.size() - 2 >= 0) {
			
			a = temporal.get(temporal.size() - 2);
			b = temporal.get(temporal.size() - 1);

			// last bigram is a special case
			String a_b = a + "_" + b;
			
			if (hyphenized.containsKey(a_b)) {

				if (suggestions.containsKey(a_b)) {
					freq = suggestions.get(a_b) + 1;

				}
				suggestions.put(a_b, freq);

			} else {

				if (suggestions.containsKey(a)) {
					freq = suggestions.get(a) + 1;

				}
				suggestions.put(a, freq);
				freq=1;
				if (suggestions.containsKey(b)) {
					freq = suggestions.get(b) + 1;

				}
				suggestions.put(b, freq);

			}

		} else {
			freq=1;
			if(temporal.size() - 1 >= 0){
				b = temporal.get(temporal.size() - 1);
				if (suggestions.containsKey(b)) {
					freq = suggestions.get(b) + 1;

				}
				
				suggestions.put(b, freq);
				
			}

		}

	}

	public LinkedList<String> splitText(String text) {

		LinkedList<String> suggestions = new LinkedList<String>();

		if (text == null) {

			return suggestions;
		}

		text = text.replaceAll("\\s+", " ").toLowerCase();
		String words[] = text.split(" ");
		for (int i = 0; i < words.length; i++) {
			
			String word = CleanTag.normalizeTag(words[i]);

			if (!cleaner.isTagTrash(word)) {
				suggestions.add(word);

			}

		}

		return suggestions;

	}

	
	
	
	public void processFile(String path, String stop_words_path, int threshold) throws IOException{
		
		
		FileInput in = new FileInput(path);
		
		String line = in.readString();
		
		while(line!=null){
			
			String t[]= line.split("\t");
			
			
			if(t.length<4){
				line = in.readString();
				continue;
				
			}
			String query = t[0];
			String url = t[1];
			String title = t[2];
			String snippet = t[3];

		
			// System.out.println(query+"\t" + title+ "\t"+ snippet);

			if (cleaner == null) {
				cleaner = new CleanTag(stop_words_path);

			}

			DataBag output = mBagFactory.newDefaultBag();

			Hashtable<String, Integer> list = getTags(query, title, snippet,
					threshold);
			
			
			if (list == null) {
				

				 System.out.println(query + "\t" + "-error-" + "\t" + (-1));

			} else if (list.size() == 0) {
			
				 System.out.println(query + "\t" + "-no_tags-" + "\t" + (-1));
			} else {
				Enumeration<String> iter = list.keys();

				while (iter.hasMoreElements()) {
					String tag = iter.nextElement();
					
					System.out.println(query+"\t"+ tag + "\t"+ list.get(tag));

				}

			}
			
			
			line  =in.readString();	
			
			
		}
		
	}
	
	public static void main(String args[]) throws IOException {
	
	//	args= new String[4];
		//args[0]="/Users/sergioduarte/projects/data/results/bing_results/bing_results_sample2_kids.txt";
		//args[1]="/Users/sergioduarte/Dropbox/results/tags_hyphenized_enriched/tags_hyphenized.enriched.txt";
		//args[2]="3";
		//args[3]="/Users/sergioduarte/Dropbox/results/tags_unigrams_clean.txt";
		
		String stop_words= "/dictionary/stop_words.txt";
		String bing_path=args[0];
		Integer threshold =Integer.valueOf(args[2]);
		String unigrams = args[3];
		String colo= args[1];
		
		GenerateQueryTagsLocal launcher = new GenerateQueryTagsLocal();
		
		
		
		launcher.processFile(bing_path, stop_words, threshold);
		
	
	}
	
}
